package com.crawler.crawler.tool.zyu;

import com.crawler.crawler.domain.Book;
import com.crawler.crawler.tool.zyu.pipeline.BookPipeline;
import com.crawler.crawler.tool.zyu.pipeline.DataPipeline;
import com.crawler.crawler.util.FileDownload;
import jdk.nashorn.internal.parser.JSONParser;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.scheduler.BloomFilterDuplicateRemover;
import us.codecraft.webmagic.scheduler.QueueScheduler;

import java.util.List;

public class ZyuBookInfo implements PageProcessor {

    private Site site = Site.me()
//            .setCharset("gbk")//设置编码
            .setTimeOut(100 * 1000)//设置超时时间
            .setRetrySleepTime(3000)//设置重试的间隔时间
            .setRetryTimes(3);//设置重试的次数

    private int lastPage;

    /**
     * process the page, extract urls to fetch, extract the data and store
     *
     * @param page page
     */
    @Override
    public void process(Page page) {
        Book book=new Book();
       String aa= page.getHtml().css("h1.kratos-entry-title").toString();
       //通过Jsoup来得到格式化
        Document document= Jsoup.parse(aa);
        //标题
        String title=document.getElementsByClass("kratos-entry-title").text();
        book.setBookName(title);
        //图片
        String imgUrl=page.getHtml().css("p>img","src").toString();
        book.setBookCover(imgUrl);

        try {
            String authorHtml=page.getHtml().xpath("//*[@id=\"main\"]/article/div[1]/div/p[3]/text()").toString();
            if(authorHtml.indexOf("【作者】 :")>=0){
                String author=authorHtml.replace("【作者】 :","");
                book.setAuthor(author);
            }
        }catch (Exception e){

        }


        String contentValidityHtml=page.getHtml().css("div.kratos-post-content").toString();
        String[] contentValidityArray=contentValidityHtml.replaceAll("<div class=\"kratos-post-content\">","").split(" <div class=\"__youshang\"> ");
        String contentValidity=contentValidityArray[0];
        book.setContentValidity(contentValidity);

        try {
            String baiduUrl=page.getHtml().css("p>a","href").toString();
            book.setBaiduUrl(baiduUrl);
            String baiduCode=contentValidity.split("提取码: ")[1].split("</p>")[0];
            book.setBaiduCode(baiduCode);
        }catch (Exception e){
            //异常不做处理
        }
        page.putField("book",book);

    }

    /**
     * get the site settings
     *
     * @return site
     * @see Site
     */
    @Override
    public Site getSite() {
        return site;
    }

    /**
     * 得到页数
     *
     * @param url
     * @return
     */
    //initialDelay当任务启动后，等等多久执行方法
    //fixedDelay每个多久执行方法
    //@Scheduled(initialDelay = 1000, fixedDelay = 100 * 1000)
    public void process(String url, BookPipeline bookPipeline) {
        Spider.create(new ZyuBookInfo())
                .addUrl(url)
                .setScheduler(new QueueScheduler().setDuplicateRemover(new BloomFilterDuplicateRemover(5000)))
                .thread(3)
                .addPipeline(bookPipeline)
                .run();
    }
}
